######## Yahoo Search Client ########
######## Final Year Project 2011#####
######## Making Web Search Spatial###


######## Liam Sullivan       ########
######## University of Leeds ########

######## Based on Tutorials found at:

######## http://www.42topics.com/dumps/appengine-search/doc.html

######## http://code.google.com/appengine/docs/python/gettingstarted/



import wsgiref.handlers
import urllib
import logging
import time

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.api import urlfetch

from google.appengine.ext import db
from google.appengine.api import users

from django.utils import simplejson
from django.utils.html import strip_tags
from operator import itemgetter
from StringIO import StringIO




## Import customfilters.py, this sits in the top level directory and contains customer filters for use within the template language

webapp.template.register_template_library('customfilters')



class Website(db.Model):
    """A class for storing classification data for a URL in the google appengine filestore"""
    url = db.StringProperty(required=True)
    Arts = db.StringProperty(required=True)
    Business = db.StringProperty(required=True)
    Science = db.StringProperty(required=True)
    Computers = db.StringProperty(required=True)
    Recreation = db.StringProperty(required=True)
    Sports = db.StringProperty(required=True)
    Society = db.StringProperty(required=True)
    Health = db.StringProperty(required=True)
    Home = db.StringProperty(required=True)
    Games = db.StringProperty(required=True)


def DebugPrint(item):
        
    print "Content-Type: text/html"
    print "<h2> " + item + " </h2>"
    
class SearchResult():
    """SearchResult objects have a Title, Extract, URL and Classification data"""
    
    def __init__(self, URL, ClickURL, Title, Extract, relevance):
        """Build object with passed in values but also generate Classification data from URL and encode in Object"""
        
        self.URL = URL
        
        self.ClickURL = ClickURL
        
        
        self.Title = Title

        
        self.Extract = Extract
        
        
        
        self.relevance = relevance
               
        #Get classification data
        self.Classification = getClassificationData(ClickURL)
        
        TopClass = ([0,"Error"])
        
        #Multiply classification values by 100 to get probabilities as %
        try:
            for key, value in self.Classification.iteritems():

                #logging.info("It was: ")
                #logging.info(value)
                NewValue = value * 100
                
                #logging.info("Its now: ")
                #logging.info(NewValue)
                
                NewValue = int(NewValue)
                
                #logging.info("Finally after rounding it is: ")
                #logging.info(NewValue)
               
                if NewValue > TopClass[0]:
                    TopClass = ([NewValue, key])
                
                #And alter the value for the key
                self.Classification[key] = NewValue
        except:
            logging.info("Problem with class data so didn't do the rounding / multiplying")
            
                        
        self.TopClass = TopClass
        
        #test - can we use an index, is class data okay?  
        try:
           
            
            self.Arts = self.Classification['Arts']
            self.Business = self.Classification['Business']
            self.Science = self.Classification['Science']
            self.Computers = self.Classification['Computers']
            self.Recreation = self.Classification['Recreation']
            self.Sports = self.Classification['Sports']
            self.Society = self.Classification['Society']
            self.Health = self.Classification['Health']
            self.Home = self.Classification['Home']  
            self.Games = self.Classification['Games'] 
        except:
            """Provide dummy info if something went wrong"""
            self.Arts = 0
            self.Business = 0
            self.Science = 0
            self.Computers = 0
            self.Recreation = 0
            self.Sports = 0
            self.Society = 0
            self.Health = 0
            self.Home = 0
            self.Games = 0
            
           
           
class MainPage(webapp.RequestHandler):
    def get(self):
        
        MainError = True
        
        #Write the header out
        
        self.response.headers['Content-Type'] = 'text/html'
        
        #Get the query from the text box which is passed in through the RequestHandler
        query = self.request.get('q', '')
        logging.info("Query is:  " + query)
        
        
        #Get the value of the radio set for command choice
        radio = self.request.get('radio', '')         
        logging.info("Radio selection is :  " + radio)        
     
        
        #Get number of results and int it
        
        NumberOfResults = self.request.get('number', '')
        
        #If there is already a query string then we'll print result        

        
        if radio == "traditional":
            
          """ Present a classification map for for all results, showing distance from topics"""
          
          # Flag off to prevent error page
          MainError = False
          
          
          #Get number of results and int it
          
          logging.info("Should do traditional")
          
          NumberOfResults = self.request.get('number', '')
          
          logging.info("Number of results requested: " + NumberOfResults)
          
          logging.debug('Query is : %s'% query)
         # results = get_search_results('Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-', query,'','',NumberOfResults)
          #results = results['Result']
          
          #new method
          appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
          
          ResultObjectsList = getTraditionalSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
          
          Traditional = True
                    
          resp = template.render('classicsearch.html', locals())     
         
        if radio == "classify":
            
          """ Present a classification map for for all results, showing distance from topics"""
          
          # Flag off to prevent error page
          MainError = False
          
          
          #Get number of results and int it
          
          NumberOfResults = self.request.get('number', '')
          
          logging.info("Number of results requested: " + NumberOfResults)
          
          logging.debug('Query is : %s'% query)
         # results = get_search_results('Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-', query,'','',NumberOfResults)
          #results = results['Result']
          
          #new method
          appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
          
          ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
          
          Classify = True
          
          resp = template.render('classify.html', locals())
        
        if radio == "spatial":
            
            """ Present a classification map for for all results, showing distance from topics"""
            
            # Flag off to prevent error page
            MainError = False
            
       
            
            TopFiveResults = []
                    
            #Get number of results and int it
            
            NumberOfResults = self.request.get('number', '')
            
            logging.info("Number of results requested: " + NumberOfResults)
            
            logging.debug('Query is : %s'% query)
            
            appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
            
            ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
            
            Classify = True
            
            
            
            def getFirstFiveElements(list):
                """Returns a list with only the first 5 entries in the input list"""
                
                outlist = []
                
                if len(list) >4:
                
                    for i in range(0,5):
                        outlist.append(list[i])
                        
                    return outlist
                
                else:
                    return list
                    
            
            
            def getTopicCounts(ResultObjectsList):
                """Takes a list of result objects and returns a dictionary with keys as the topics and values giving the number of results classified as that topic, in this initial view this is based on the largest (top) class from each result"""
                
                #initialise the dictionary to hold counts
                
                output = {}
                output["Arts"] = 0
                output["Business"] = 0
                output["Games"] = 0
                output["Home"] = 0
                output["Health"] = 0
                output["Society"] = 0
                output["Sports"] = 0
                output["Recreation"] = 0
                output["Computers"] = 0
                output["Science"] = 0
                
                for res in ResultObjectsList:
                    ThisTopClass = str(res.TopClass[1])
                    if ThisTopClass == "Error":                        
                        logging.info("Error with class data so don't count this one")
                    else:
                        output[ThisTopClass] = int(output[ThisTopClass]) + 1
                        logging.info("Incremented the count for that class" + str(ThisTopClass))
                        
                return output
                    
                                        
            TopicCounts = getTopicCounts(ResultObjectsList)
            logging.info("Got Topic counts for passing to control")
            

            ResultsArts = []
            ResultsBusiness = []
            ResultsGames = []
            ResultsHome = []
            ResultsHealth = []
            ResultsSociety = []
            ResultsSports = []
            ResultsRecreation = []
            ResultsComputers = []
            ResultsScience = []
               
            #Sort the list of result objects by relevance so all topic results lists will be in order also
            
            ##There already in order!!!
            
            # ResultObjectsList.sort(key = lambda x: x.relevance)
            
            
            #Get info about the results list
            for res in ResultObjectsList:
                
                if res.relevance <6:
                    
                    TopFiveResults.append(res)
                
                TopClass = str(res.TopClass[1])
                
                if TopClass == "Arts":
                    ResultsArts.append(res)
                    
                if TopClass == "Business":
                    ResultsBusiness.append(res)
                    
                if TopClass == "Games":
                    ResultsGames.append(res)
                    
                if TopClass == "Home":
                    ResultsHome.append(res)
                    
                if TopClass == "Health":
                    ResultsHealth.append(res)
                    
                if TopClass == "Society":
                    ResultsSociety.append(res)
                    
                if TopClass == "Sports":
                    ResultsSports.append(res)
                    
                if TopClass == "Recreation":
                    ResultsRecreation.append(res)
                    
                if TopClass == "Computers":
                    ResultsComputers.append(res)
                    
                if TopClass == "Science":
                    ResultsScience.append(res)
                    
            FiveResultsArts = getFirstFiveElements(ResultsArts)  
            
            #A list to hold results grouped by topic
            
            ResultsByTopic = []            
     
            #Populate it
            ResultsByTopic.append(ResultsArts)
            ResultsByTopic.append(ResultsBusiness)
            ResultsByTopic.append(ResultsGames)
            ResultsByTopic.append(ResultsHome)
            ResultsByTopic.append(ResultsHealth)
            ResultsByTopic.append(ResultsSociety)
            ResultsByTopic.append(ResultsSports)
            ResultsByTopic.append(ResultsRecreation)
            ResultsByTopic.append(ResultsComputers)
            ResultsByTopic.append(ResultsScience)
            
            

            FiveResultsBusiness = getFirstFiveElements(ResultsBusiness)
            FiveResultsGames = getFirstFiveElements(ResultsGames)
            FiveResultsHome = getFirstFiveElements(ResultsHome)
            FiveResultsHealth = getFirstFiveElements(ResultsHealth)
            FiveResultsSociety = getFirstFiveElements(ResultsSociety)
            FiveResultsSports = getFirstFiveElements(ResultsSports)
            FiveResultsRecreation = getFirstFiveElements(ResultsRecreation)
            FiveResultsComputers = getFirstFiveElements(ResultsComputers)
            FiveResultsScience = getFirstFiveElements(ResultsScience)
            
            
            
                
            #Encode what we find into the JSON to drive the chart
            
            def formatSize(size):
                """To make the donut more useful times all by 100 and add 1 to make everything have a size"""
                
                if size>0:
                    outsize = size * 100
                    outsize = outsize + 100
                
                    return outsize
            
                else:
                    return size
            
            #Take a list of results and distribute it into 10 lists by topic
  
            
            DjangoData = {
                          
                          
    "children": [                
       {
        
                 "children": [ 
                            
                            
        

                       
                       
                       
                       
                       
                       
          
      
       
                              
                              
                    ], 
         
         
         "data": {
           "description": "B", 
           "$color": "#B0AAF6", 
           "days": int(TopicCounts['Science']), 
           "$angularWidth": formatSize(int(TopicCounts['Science'])), 
           "size": 13880
         }, 
         "id": "Topics/Science", 
         "name": "Science"
       },
       
       
       
       
       
       






       {
         "children": [
                      
                      
                      

                
                
       
                

                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": int(TopicCounts['Business']), 
           "$angularWidth": formatSize(int(TopicCounts['Business'])), 
           "size": 35549
         }, 
         "id": "Topics/Business", 
         "name": "Business"
       },  
       







       {
         "children": [
                      
                      
      
                
                
   
                

                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#B0AAF6", 
           "days": int(TopicCounts['Computers']), 
           "$angularWidth": formatSize(int(TopicCounts['Computers'])), 
           "size": 35549
         }, 
         "id": "Topics/Computers", 
         "name": "Computers"
       }, 
       
       
       
       


   
       {
         "children": [
                      

    
                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
           "data": {
           "description": "B", 
         
           "$color": "#3366CC", 
           "days": int(TopicCounts['Society']), 
           "$angularWidth": formatSize(int(TopicCounts['Society'])), 
           "size": 35549
         }, 
         "id": "Topics/Society", 
         "name": "Society"
       }, 
       
       




   
       {
         "children": [
                      
                      
          
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
           "data": {
           "description": "B", 
           "$color": "#B2ABF4", 
           "days": int(TopicCounts['Sports']), 
           "$angularWidth": formatSize(int(TopicCounts['Sports'])), 
           "size": 35549
         }, 
         "id": "Topics/Sports", 
         "name": "Sports"
       }, 
       
       
       
       
 

   
       {
         "children": [
                      
                      
    
         
                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": int(TopicCounts['Games']), 
           "$angularWidth": formatSize(int(TopicCounts['Games'])), 
           "size": 50
         }, 
         "id": "Topics/Games", 
         "name": "Games"
       }, 
       
       
       
       
             
   
 

   
       {
         "children": [
                      
                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#B0AAF6", 
           "days": int(TopicCounts['Recreation']), 
           "$angularWidth": formatSize(int(TopicCounts['Recreation'])), 
           "size": 87820
         }, 
         "id": "Topics/Recreation", 
         "name": "Recreation"
       }, 
       
       
       
       
       
      
   
 

   
       {
         "children": [
                      
                      
         
                
          
                
                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": int(TopicCounts['Health']), 
           "$angularWidth": formatSize(int(TopicCounts['Health'])), 
           "size": 87820
         }, 
         "id": "Topics/Health", 
         "name": "Health"
       }, 
       
       
       
       
            
       
       





    
       
       
 

   
       {
         "children": [
                      

                       
                       
                       
                       
                       
                       
          
         ], 
         
         
         
         
         
         
         
         
         
         "data": {
           "description": "B", 
           "$color": "#B2ABF4", 
           "days": int(TopicCounts['Home']), 
           "$angularWidth": formatSize(int(TopicCounts['Home'])), 
           "size": 32900
         }, 
         "id": "Topics/Home", 
         "name": "Home"
       }, 
       
       

       
       
 

   
       {
         "children": [
                       
                       
                       
                       
          
         ], 
         
    
       
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": int(TopicCounts['Arts']), 
           "$angularWidth": formatSize(int(TopicCounts['Arts'])), 
           "size": 186221
         }, 
         "id": "Topics/Arts", 
         "name": "Arts"
       }
     ], 
     "data": {
       "$type": "none"
     }, 
     "id": "Topics", 
     "name": "All Topics"
   }


            def addTopResultFlag(DjangoData, Topic):
                
                """Adds an entry in the viz for the top result, to mark out which topic houses the most relevant result from the query"""
                
                #Inject top result!
                
                ##Maybe need to have them hidden and display if they are needed
                
                
                Injection =   {
                               
                 
                           "children": [],                            
                           
                           
                           "data": {  
 
                                 "$angularWidth": 1,   
                                 "$color":"#000000",   

                                    }
                           
                           ,                             
                            
                           "id": "Top Result",   
                           "name": "Top Result"
                
    
                                }
            
                
                if Topic == "Science":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][00]['children'].append(Injection)                    
                    
                
                if Topic == "Business":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][01]['children'].append(Injection)
                    
                if Topic == "Computers":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][02]['children'].append(Injection)
                    
                if Topic == "Society":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][03]['children'].append(Injection)
                    
                    
                if Topic == "Sports":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][04]['children'].append(Injection)
                    
                if Topic == "Games":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][05]['children'].append(Injection)
                    
                if Topic == "Recreation":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][06]['children'].append(Injection)
                    
                    
                if Topic == "Health":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][07]['children'].append(Injection)
                    
                    
                if Topic == "Home":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][8]['children'].append(Injection)
                    
                if Topic == "Arts":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][9]['children'].append(Injection)
                    

                    
                return DjangoData
            
            
               
            def addOuterLayer(DjangoData,Topic, Label, Size, Colour):
                
                """Adds the outer layer to a topic, giving relevance breakdown"""      
                
                if Label == "none":
                    Label = " "
                
                Size = int(Size)
                
                ID = "Topics/" + str(Topic) + "/" + str(Label)          
                
                Injection =       {
                
                 "children": [],                            
                           
                           
                           "data": {  
 
                                 "$angularWidth": Size,   
                                 "$color": Colour,    

                                    }
                           
                           ,                             
                            
                           "id": ID,   
                           "name": Label
                
                                               
                               
                               
                               
                               }
                
                if Topic == "Science":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][00]['children'].append(Injection)                    
                    
                
                if Topic == "Business":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][01]['children'].append(Injection)
                    
                if Topic == "Computers":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][02]['children'].append(Injection)
                    
                if Topic == "Society":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][03]['children'].append(Injection)
                    
                    
                if Topic == "Sports":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][04]['children'].append(Injection)
                    
                if Topic == "Games":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][05]['children'].append(Injection)
                    
                if Topic == "Recreation":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][06]['children'].append(Injection)
                    
                    
                if Topic == "Health":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][07]['children'].append(Injection)
                    
                    
                if Topic == "Home":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][8]['children'].append(Injection)
                    
                if Topic == "Arts":
                    
                    #Inject the Top Result entry!
                    DjangoData['children'][9]['children'].append(Injection)
                    

                    
                return DjangoData
            
            
            #What is the top result?
            
            FirstResult = TopFiveResults[0]
            
            FirstResultTopic = FirstResult.TopClass[1]
            
            #Add the top result flag to the wheel
            
            #Feature disabled, due to encoding confusion
            #Discussed in Design chapter
            
            #DjangoData = addTopResultFlag(DjangoData, FirstResultTopic)
            
            
           # def getTopTenPercentage(Topic):
             #   """Takes results set for a given topic and returns stats to build relevance breakdown"""
                
               # for result in Topic:
                    
                
               # return output

                
                
            for Topic in ResultsByTopic:
                
                if len(Topic)>0:
                    
                    TopicIn = Topic[0].TopClass[1]

                    
                    FirstTenResultsCount = 0
                    NextTwentyResultsCount = 0
                    LowerRelevanceResultsCount = 0
                    
                    #How many results in the top ten?
                    
                    for result in Topic:
                        
                        if result.relevance <11:
                            FirstTenResultsCount = FirstTenResultsCount + 1
                            
                        elif result.relevance <31:
                            NextTwentyResultsCount = NextTwentyResultsCount + 1
                            
                        else:
                            LowerRelevanceResultsCount = LowerRelevanceResultsCount + 1
                            
                    logging.info("So out of " + str(len(Topic)) + " results there are: " + str(FirstTenResultsCount) + " results in the Top Ten and: " + str(NextTwentyResultsCount) + " in the next twenty most relevant, with the rest of the results in this topic being out of the top 30")


                    #Add breakdown top 10 element
                    
                    DjangoData = addOuterLayer(DjangoData,TopicIn, "High Relevance", FirstTenResultsCount, "336600")
                    
                    
                    #Add breakdown next twenty element
                    DjangoData = addOuterLayer(DjangoData,TopicIn, "Mid Relevance", NextTwentyResultsCount, "FF9900")
                    
                    #Add breakdown all otheres (low relevance)
                    
                    DjangoData = addOuterLayer(DjangoData,TopicIn, "Low Relevance", LowerRelevanceResultsCount, "FF0000")
                    
                    

                    
                    
                    
                else:
                    logging.info("Nothing in that topic so leave it without any outer layer entry")
            
            #Loop through a list of results by topics
            
            #if topic length > 0
            
            #add breakdown elements to graph
            
            #if not ignore           
            
            
            resp = template.render('SpatialSearch.html', locals())
            
        if radio == "naivespatial":
            
            """Naive presentation of classifier data"""
            
            # Flag off to prevent error page
            MainError = False
                    
            #Get number of results and int it
            
            NumberOfResults = self.request.get('number', '')
            
            logging.info("Number of results requested: " + NumberOfResults)
            
            logging.debug('Query is : %s'% query)
            
            appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
            
            ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
            
            Classify = True
            
            resp = template.render('NaiveSpatialSearch.html', locals())
            
        if radio == "offline":
            """A branch to allow working when the classifier is down"""
            
            # Flag off to prevent error page
            MainError = False
            
            logging.info("Problems with an external source so going to offline / testbed / error page")
            
            resp = template.render('offline.html', locals())
            
            
        #Final case is an error
            
        if MainError:
            logging.info("Either landed on form for first time or something went wrong")
            resp = template.render('form.html', locals())
            
        #Regardles of mode, we respond with something to drive page creation  
        self.response.out.write(resp)


def main():
  application = webapp.WSGIApplication(
                                       [('/', MainPage)],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)



def get_search_results(appid, query, region ='us', type = 'all', results = 2, start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    result = _query_yahoo(base_url, params)
    return result['ResultSet']

def _query_yahoo(base_url, params):
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    result = simplejson.load(response)
    return result


def getTraditionalSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    logging.info("Calling service with URL: ")
    logging.info(url)
    result = simplejson.load(response)
    result = result['ResultSet']
    AllResults = result['Result']
    
    ResultObjectList = []
    
    ResultRank = 0
    
    for res in AllResults:
        """Loop through all results, they are in order of relevance"""
        
        #Increment ResultRank
        ResultRank = ResultRank + 1
        
        #Add relevance
        res['Relevance'] = ResultRank
        
        #Don't do any classification as this is just traditional search
        
        #Add it to the list
        ResultObjectList.append(res)
    
    return ResultObjectList
    
    
def getSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    logging.info("Calling service with URL: ")
    logging.info(url)
    result = simplejson.load(response)
    result = result['ResultSet']
    AllResults = result['Result']
    
    ResultObjectList = []
    
    ResultRank = 1
    
    for res in AllResults:
        """Loop through all results, they are in order of relevance"""
        
        
        
        logging.info(res)
        
        #Make a new SearchResult Object
        SearchResultInstance = SearchResult(res['Url'], res['ClickUrl'], res['Title'], res['Summary'], ResultRank)
        
        #Increment ResultRank
        ResultRank = ResultRank + 1
        
        #Use the object method to fetch classification data before saving object
        
        
        #Add it to the list
        ResultObjectList.append(SearchResultInstance)
    
    return ResultObjectList


def getClassificationData(URL):
    """Classifies a URL using the UClassify Service"""
    
    
    ##Set maximum attempts here
    
    MaximumAttempts = 5
    
    #Dictionary containing all params, which we will encode for REST Call    
    Params = {}   
        
    #BaseURL
    #uClassify URL Endpoint
    endpoint = "http://uclassify.com/browse/uClassify/Topics/ClassifyUrl?"
    
    #API Key    
    Params['readkey'] = "TsE3A35FCBSAdbGFLEvI2PXH0"
    
    #Output Format
    Params['output'] = "json"
    
    #Incoming URL that needs classifying
    Params['url'] = URL
    
    #Version
    Params['version'] = 1.01
    
    #Strip HTML
    Params['removeHtml'] = True
    
    #Encode Params from dictionary for call
    
    RestParams = urllib.urlencode(Params)    
    
    #Combine base url and params
    
    RestURL = endpoint + RestParams
    
    logging.info("Trying to classify URL : ")
    logging.info(URL)
    
    
    #while not suceeded, try
    
    GotClassificationData = False
    AttemptNumber = 0
    
    
    while not GotClassificationData and not AttemptNumber > MaximumAttempts:
        
        try:
            #Try to get classifier response
        
            AttemptNumber = AttemptNumber + 1
            logging.info("Making attempt number: " + str(AttemptNumber))
            response = StringIO(urlfetch.fetch(RestURL, deadline=10).content)   
            logging.info("Attempt number " + str(AttemptNumber) + " worked.")
            GotClassificationData = True
            
        except:                        
            logging.info("Attempt number " + str(AttemptNumber) + " failed.")
            logging.info("The classifier took to long to respond ( > 10 seconds)")
            logging.info("Trying again....often this results in a faster attempt that will not timeout.")
            AttemptNumber = AttemptNumber + 1
            time.sleep(1)
                
    
    #Either got results or finished maximum tries
                
        
     
    
    try:
        result = simplejson.load(response)
        ResultSet =  result['cls1'] 
        logging.info("Results from classifier: ")
        logging.info(ResultSet)
        return ResultSet
    except:
        logging.info("Didn't get any classifier results")
    

    
    


def ClassifyURL(URL):
    
    """Classifies a URL, calls other method to get the results from the Web Service then computes highest probability and returns as a string"""

    #Pass URL and get the dictionary results from the classifier
    Results = getClassificationData(URL)
    
    #Take the first key from a sorted set of the dictionary, this is the most likely topic as classified
    
    SortedResults = sorted(Results.items(), key=itemgetter(1), reverse=True)
    
    TopResult = SortedResults[0]
    
    return TopResult[0]




if __name__ == "__main__":
    main()
